#include "../../include/RssTest.h"
#include "../../include/hashdup.h"
#include "../../include/SplitToolCppJieba.h"
#include "../../include/Logger.h"
#include "../../include/MySQLConnector.h"
#include <unistd.h>
#include <sstream>
#include <algorithm>

using std::string;

using std::istringstream;
RSS_read::RSS_read(const string &folderPath)
:_folderPath(folderPath)
{
    /* cout << "RSS_read(const string &folderPath)"<<endl; */
}
//读所有的xml文件 并进行操作
void RSS_read::read(){
    string folderPath = _folderPath;  // 替换为您的文件夹路径
    DIR* directory = opendir(folderPath.c_str());
    if (directory == nullptr) {
        LogError("Failed to open directory: %s", folderPath.c_str());
        return;
    }

    struct dirent* entry;
    while ((entry = readdir(directory)) != nullptr) {
        const string fileName = entry->d_name;
        const string filePath = folderPath + "/" + fileName;

        if (entry->d_type == DT_REG && fileName.substr(fileName.size() - 4) == ".xml") {
            XMLDocument doc;
            doc.LoadFile(filePath.c_str());
            if(doc.ErrorID()){
                LogError("read xml path error");
                break;                
            }
            LogInfo("read xml: %s", fileName.c_str());
            xmlRead(filePath);
        }
    }
    closedir(directory);
}
//正则表达式操作 取出html标签
string RSS_read::remove_html_tags(const string& text){
    std::regex pattern("<[^>]*>");
    string result = std::regex_replace(text, pattern, "");
    return result;
}

string removeNewlinesAndTabs(const string& input) {
    std::regex whitespaceRegex("[\\t\\n\\r ]+");  // 匹配制表符、换行符和多个空格
    return std::regex_replace(input, whitespaceRegex, " ");
}

void removeLessThanSigns(std::string& sentence) {
    size_t pos = sentence.find('<');
    while (pos != std::string::npos) {
        sentence.erase(pos, 1);  // 移除找到的 '<'
        pos = sentence.find('<', pos); // 继续查找下一个 '<'
    }
}

void RSS_read::xmlRead(const string &filePath){
    XMLDocument doc;
    doc.LoadFile(filePath.c_str());
    if(doc.ErrorID()){
        LogError("xml read error");
        return;
    }
    XMLElement *itemNode = doc.FirstChildElement("rss")->FirstChildElement("channel")->FirstChildElement("item");
        while(itemNode){
        RssiTeam rss;
        bzero(&rss,sizeof(rss));

        string title;
        string link;
        string description;
        string content;

        XMLElement * ptitle = itemNode->FirstChildElement("title");
        //判断是否有title
        if(ptitle){
            //获取text
            title = ptitle->GetText();
            title = remove_html_tags(title);
            if(title.empty()){
                itemNode = itemNode->NextSiblingElement("item");
                continue;
            }
        }
        XMLElement * plink = itemNode->FirstChildElement("link");
        if(plink){
            //获取text
            link  = plink->GetText();
            if(link.empty()){
                itemNode = itemNode->NextSiblingElement("item");
                continue;
            }
        }

        XMLElement * pdescription = itemNode->FirstChildElement("description");
        XMLElement * pcontent = itemNode->FirstChildElement("content");
        if(pdescription || pcontent){
            //如果存在一个 就进行记录
            if(pdescription && pdescription->GetText()!=NULL){
                //cout<<"75 我执行拉"<<endl;
                description = pdescription->GetText();
                description = remove_html_tags(description);
                if (!description.empty() && description.find_first_not_of(' ') == std::string::npos) {
                // description 的长度不为空，并且其值只包含空格字符的处理逻辑
                    itemNode = itemNode->NextSiblingElement("item");
                    continue;
                }
                if(description.empty()){
                    rss._description = "0";
                    itemNode = itemNode->NextSiblingElement("item");
                    continue;
                }
            }else{
                description = "0";
            }
            if(pcontent && pcontent->GetText()!=NULL){
                content = pcontent->GetText();
                content = remove_html_tags(content);
                if(content.empty()){
                    rss._content = "0";
                    itemNode = itemNode->NextSiblingElement("item");
                    continue;
                }
            }else{
                content = "0";
            }
        }
        else{
            //如果都没有 就全部跳过
            itemNode = itemNode->NextSiblingElement("item");
            continue;
        }

        title = remove_html_tags(title);
        removeLessThanSigns(title);
        rss._title = title;

        link = remove_html_tags(link);
        removeLessThanSigns(link);
        rss._link = link;
        if (description == "0") {
            rss._description = description;
        }
        else{
            description = remove_html_tags(description);
            removeLessThanSigns(description);
            rss._description = description;
        }
        if (content == "0") {
            rss._content = content;
        }
        else{
            content = remove_html_tags(content);
            removeLessThanSigns(content);
            rss._content = content;
        }
        rss._text = rss._title + rss._link + rss._description + rss._content;
        _RSS.push_back(rss);
        itemNode = itemNode->NextSiblingElement("item");
    }
}
//get Hash
//计算哈希值
uint64_t RSS_read::getHash(const string &s1) {
    size_t topN = 10;
    uint64_t u64 = 0;
    vector<pair<string, double>> res;
    //问题
    Hashdup::getpHashdup()->simhasher.extract(s1, res, topN);
    Hashdup::getpHashdup()->simhasher.make(s1, topN, u64);
    return u64; // 返回哈希值
}

//循环遍历去计算hash
void RSS_read::forHash(){
    for(size_t i = 0;i != _RSS.size();i++){
        _RSS[i]._hash = getHash(_RSS[i]._text);
    }
}
//对hash的处理
//移动语义可以优化

void RSS_read::removeSame() {
    forHash();
    /* cout << "执行 removeSame" << endl; */
    vector<size_t> saveHash; //存saveHash
    vector<RssiTeam> temp; //存新的rss
    //先方一个
    saveHash.push_back(_RSS[0]._hash);
    //存放hash 
    for(size_t i = 1 ;i != _RSS.size();++i){
        //进行比较
        //如果不相等
        if(Hashdup::getpHashdup()->simHash1(_RSS[i-1]._hash,_RSS[i]._hash)==0){
            temp.push_back(_RSS[i]);
            //hash 也要放
            saveHash.push_back(_RSS[i]._hash);
        }
    }
    _RSS = temp;
}

string escapeSingleQuotes(const string& input) {
    string output;
    for (char c : input) {
        if (c == '\'') {
            output += '\\';
        }
        output += c;
    }
    return output;
}

void RSS_read::store(const string &pagelib, const string &offset)
{
    ofstream ofs_webpage(pagelib);
    ofstream ofs_offset(offset);

    if(!ofs_webpage || !ofs_offset)
    {
        LogError("Failed to open %s / %s!", pagelib.c_str(), offset.c_str());
        return;
    }

    stringstream ss;
    size_t offsetIdx = 0;
    int docid = 0;

    LogInfo("read xml size: %zu", _RSS.size());

    MySQLConnector *con = MySQLConnector::getInstance();
    /* string insertQuery = "insert into t_webpage (article_id, title, link, content, description) values"; */
    string insertQuery;

    for(size_t idx = 0; idx != _RSS.size(); ++idx)
    {
        docid = idx;
        ss << "<doc>\n" 
            << "\t<docid>" << docid + 1 << "</docid>\n"
            << "\t<title>" << _RSS[idx]._title << "</title>\n"  
            << "\t<link>" << _RSS[idx]._link << "</link>\n" 
            << "\t<description>" << _RSS[idx]._description << "</description>\n"
            << "\t<content>" << _RSS[idx]._content << "</content>\n"
            << "</doc>\n";

        string description = escapeSingleQuotes(_RSS[idx]._description);
        string content = escapeSingleQuotes(_RSS[idx]._content);

        if("0" == description) {
            description = "";
        }

        if("0" == content) {
            content = "";
        }

        string s1 = "insert into t_webpage (article_id, title, link, content, description) values";
        insertQuery = s1  +  "('" + std::to_string(docid + 1) + "', '" + _RSS[idx]._title + "', '" + _RSS[idx]._link + "', '" + content + "', '" + description + "');";

        con->executeQuery(insertQuery);

        string tmp = ss.str();
        ss.str("");
        ofs_webpage << tmp;
        ofs_offset << docid + 1 << " " << offsetIdx << " " << tmp.size() + offsetIdx - 1 << endl;
        offsetIdx += tmp.size();    
    }


    ofs_offset.close();
    ofs_webpage.close();
}

